#!/bin/bash

	
	outputfile=data_to_insert
	exec 3> $outputfile
	>$outputfile #clear the file

	errorfile=errors.out
	exec 2> $errorfile	
	>$errorfile #clear the file

## Usage ##

	USAGE="Usage: $0 [-orv] [settings_file] target_directory"

## END OF Usage ##

## Parameter flags ##

	oflag=
	vflag=
	rflag=
	settingsfile=
	searchpath=

## END OF Parameter flags ##

## Settings parameters ##

	# Default settings
	alpha_max_line_length=80
	alpha_max_word_length=20
	numerical_max_value=99999
	numerical_min_value=0

## END OF Settings parameters ##

## Statistics variables ##

	# Statistics
	files_processed_total=0
	files_processed_alpha_total=0
	files_processed_alpha_pct=0
	files_processed_num_total=0
	files_processed_num_pct=0
	total_items_processed=0
	items_errors_total=0
	items_errors_pct=0

	# Alphanumerical variables
	lines_processed=0
	lines_out_of_bounds_total=0
	lines_out_of_bounds_pct=0
	words_processed=0
	words_out_of_bounds_total=0
	words_out_of_bounds_pct=0
	lines_length_total=0
	longest_line_length=0
	shortest_line_length=
	average_line_length=0
	median_line_length=0

	# Numerical variables
	numbers_processed_total=0
	numbers_out_of_bounds_total=0
	numbers_out_of_bounds_pct=0
	numbers_out_of_bounds_high_total=0
	numbers_out_of_bounds_high_pct=0
	numbers_out_of_bounds_low_total=0
	numbers_out_of_bounds_low_pct=0
	highest_value=0
	lowest_value=
	number_value_total=0
	average_value=0
	median_value=0

	# Errors
	erroneous_lines=0
	erroneous_words=0
	erroneous_numbers_high=0
	erroneous_numbers_low=0

## END OF Statistics variables ##

## Functions ##

	function get_pct() {
		div=$2
		if [ $div == 0 ]; then
			div=1;
		fi
		echo "$(echo "scale=2;$1*100/$div"|bc)%"
	} ## END OF get_pct()

	function get_div() {
		div=$2
		if [ $div == 0 ]; then
			div=1;
		fi
		echo "$(echo "$1/$div"|bc)%"
	} ## END OF get_pct()

	function echo2 {
		echo -e "$1"
	}

	function create_stats() {
		files_processed_total=$(($files_processed_alpha_total + $files_processed_num_total))
		files_processed_alpha_pct=$(get_pct $files_processed_alpha_total $files_processed_total)
		files_processed_num_pct=$(get_pct $files_processed_num_total $files_processed_total)
		lines_out_of_bounds_pct=$(get_pct $lines_out_of_bounds_total $lines_processed)

		words_out_of_bounds_pct=$(get_pct $words_out_of_bounds_total $words_processed)
		average_line_length=$(get_div $lines_length_total $((lines_processed - lines_out_of_bounds_total)))
		numbers_out_of_bounds_total=$(($numbers_out_of_bounds_high_total + $numbers_out_of_bounds_low_total))
		numbers_out_of_bounds_pct=$(get_pct $numbers_out_of_bounds_total $numbers_processed_total) 
		numbers_out_of_bounds_high_pct=$(get_pct $numbers_out_of_bounds_high_total $numbers_processed_total)
		numbers_out_of_bounds_low_pct=$(get_pct $numbers_out_of_bounds_low_total $numbers_processed_total)
		average_value=$(get_div $number_value_total $((numbers_processed_total - numbers_out_of_bounds_total)))
		total_items_processed=$(($lines_processed + $words_processed + $numbers_processed_total))
		items_errors_total=$(($lines_out_of_bounds_total + $words_out_of_bounds_total + $numbers_out_of_bounds_total))
		items_errors_pct=$(get_pct $items_errors_total $total_items_processed)

		# Total files processed
		echo2 "Total files processed: $files_processed_total\n"

		# Alphanumerical files # and %
		echo2 "Total alphanumerical files processed: $files_processed_alpha_total" 
		echo2 " Percentage of alphanumerical files: $files_processed_alpha_pct\n"

		# Numerical files # and %
		echo2 "Total numerical files processed: $files_processed_num_total" 
		echo2 " Percentage of numerical files: $files_processed_num_pct\n" 

		# Total number of items
		echo2 "Total number of items processed: $total_items_processed" 
		echo2 " Total number of items errors: $items_errors_total" 
		echo2 " Percentage of total number of item errors : $items_errors_pct\n" 

		# Lines processed
		echo2 "Total number of lines processed: $lines_processed" 
		echo2 " Lines out of bounds: $lines_out_of_bounds_total" 
		echo2 " Percentage of lines out of bounds: $lines_out_of_bounds_pct" 
		echo2 " Length of longest line: $longest_line_length" 
    	echo2 " Length of shortest line: $shortest_line_length" 
    	echo2 " Average line length: $average_line_length\n" 
    #echo -e " Median line length: $median_line_length\n" # TODO

		# Words processed
		echo2 "Total number of words processed: $words_processed" 
		echo2 " Words out of bounds: $words_out_of_bounds_total" 
		echo2 " Percentage of words out of bounds: $words_out_of_bounds_pct\n" 
		
		# Number stats
		echo2 "Total numbers processed: $numbers_processed_total" 
		echo2 " Numbers out of bounds: $numbers_out_of_bounds_total" 
		echo2 " Percentage of numbers out of bounds: $numbers_out_of_bounds_pct"
		echo2 " Numbers above threshold: $numbers_out_of_bounds_high_total" 
		echo2 " Percentage of numbers above threshold: $numbers_out_of_bounds_high_pct" 
		echo2 " Numbers below threshold: $numbers_out_of_bounds_low_total" 
		echo2 " Percentage of numbers below threshold: $numbers_out_of_bounds_low_pct" 
		echo2 " Highest value: $highest_value" 
		echo2 " Lowest value: $lowest_value" 
		echo2 " Average value: $average_value" 

	} ## END OF create_stats()

	function setsettings() {
		# Check if file exists and is readable
		if [ -r $1 ]; then
			settingsfile=$1
		else 
			echo "Settings file not found" 1>&2
		fi 	
	} ## END OF setsettings() ##

	function setsearchpath() {
		# Check if directory exists
		if [ -d $1 ]; then
			searchpath=$1
		else
			echo "Search path not found" 1>&2
		fi	
	} ## END OF setsearchpath() ##

	function parseoptions() {

		while getopts ":ovr" opt; do
			case $opt in
				o)
					oflag=1
					;;
				v)
					vflag=1
					;;
				r)
					rflag=1
					;;
				\?)
					echo "Invalid option: -$OPTARG" 1>&2
					;;
			esac
		done

		shift $((OPTIND-1))
		args=("$@")

		if [ "$#" == "1" ]; then
			setsearchpath "${args[0]}"
		elif [ "$#" == "2" ]; then
			setsettings "${args[0]}"
			setsearchpath "${args[1]}"
		else
			echo $USAGE 1>&2
	    exit 1
		fi
	} # END OF parseoptions
	
	function read_settings() {

		while read line 
	    do
				if [[ "$line" =~ "alpha_max_line_length=" ]]; then
				 	alpha_max_line_length=${line##*=}
				fi
				if [[ "$line" =~ "alpha_max_word_length=" ]]; then
				 	alpha_max_word_length=${line##*=}
				fi
				if [[ "$line" =~ "numerical_max_value=" ]]; then
				 	numerical_max_value=${line##*=}
				fi
				if [[ "$line" =~ "numerical_min_value=" ]]; then
				 	numerical_min_value=${line##*=}
				fi
	    done < "$1"
	} ## END OF read_settings() ##

	function process_numerical() {
			# Add one to number of numerical files processed
			files_processed_num_total=$(($files_processed_num_total + 1))
	    
	    words="$(grep -Po "(\s|\-)*\w+" $f)"

	    for word in $words 
	    do

				numbers_processed_total=$(($numbers_processed_total + 1))

				if [ $word -lt $numerical_min_value ]; then
		    		echo "out of bounds, number to low: $word" 1>&2
					numbers_out_of_bounds_low_total=$(($numbers_out_of_bounds_low_total + 1))		    		
					continue
				fi				
				if [ $word -gt $numerical_max_value ]; then
		    		echo "out of bounds, number to hight: $word" 1>&2
					numbers_out_of_bounds_high_total=$(($numbers_out_of_bounds_high_total + 1))
		    		continue
				fi
				
				if [ ! -z "$oflag" ]; then
					echo "insert into weird_numbers ( source, weird_number ) values ( ‘$f’, $word);" 1>&3
					if [ ! -z "$vflag" ]; then
						echo "insert into weird_numbers ( source, weird_number ) values ( ‘$f’, $word);"
					fi
				fi

				if [ -z $lowest_value ]; then
					lowest_value=$word
     			fi
     			if [ $word -lt $lowest_value ]; then
       			lowest_value=$word
     			fi
				if [ $word -gt $highest_value ]; then
       			highest_value=$word
     			fi

				number_value_total=$(($number_value_total + $word))
			done
	} ## END OF process_numerical() ##

	function process_alphanumerical() {
		# Add one to number of alphanumerical files processed
		files_processed_alpha_total=$(($files_processed_alpha_total + 1))

		 typeset line_number
		 line_number=0
	    while read line
	    do
				line_number=$((line_number + 1))
				lines_processed=$(($lines_processed + 1))

				#stats
				if [ ${#line} -gt $longest_line_length ]; then
					longest_line_length=${#line}
				fi
				if [ -z $shortest_line_length ]; then
					shortest_line_length=${#line}
				fi
				if [ ${#line} -lt $shortest_line_length ]; then
          		shortest_line_length=${#line}
        		fi
				lines_length_total=$(($lines_length_total + ${#line}))

				#sql
				if [ ${#line} -gt $alpha_max_line_length ]; then
		    		echo "Out of bounds, line too long: $line" 1>&2
					lines_out_of_bounds_total=$(($lines_out_of_bounds_total + 1))
					continue
				elif [ ${#line} -eq 0 ]; then
					continue
				else
					if [ ! -z "$oflag" ]; then
						echo "insert into lines ( source, line ) values( ‘$1’, ‘$line’ );" 1>&3
						if [ ! -z "$vflag" ]; then
							echo "insert into lines ( source, line ) values( ‘$1’, ‘$line’ );"
						fi
					fi
					typeset words
					words="$(echo $line | grep -Po "[a-zA-Z]+")"
					for word in $words 
					do
						if [ ${#word} -gt $alpha_max_word_length ]; then
							echo "Out of bounds, word too long: $word" 1>&2
							words_out_of_bounds_total=$(($words_out_of_bounds_total + 1))
						fi
						if [ ! -z "$oflag" ]; then
							echo "insert into words ( source, line_number, word ) values( ‘$1’, $line_number, ‘$word’);" 1>&3
							if [ ! -z "$vflag" ]; then
								echo "insert into words ( source, line_number, word ) values( ‘$1’, $line_number, ‘$word’);"
							fi
						fi

					done
				fi
	    done < "$1"
	} ## END OF process_alphanumerical() ##

	chunksize=2500  # Optimal
	#chunksize=25
	function travarse_the_search_path() {
	    
	  for f in $(find $1 -type f)
	  do
			while IFS= read -r -n$chunksize chunk
			do
    		if  echo "$chunk" | grep -Po "[a-zA-Z]+" > /dev/null; then
					process_alphanumerical $f
		    else
					process_numerical $f
		    fi
		    break  #just read the first line, should be enough
			done < "$f"
		done 
	} ## END OF travarse_the_search_path() ##


	function actonparams() {
		if [ ! -z "$settingsfile" ]; then
		  read_settings $settingsfile
		fi
		if [ ! -z "$vflag" ]; then
			set -x
		fi
		if [ ! -z "$searchpath" ]; then
		  travarse_the_search_path $searchpath
		fi
		
		if [ ! -z "$rflag" ]; then
			create_stats
		fi

	} # END OF actonparams

## END OF Functions ##

## Main ##

	parseoptions "${@}"
	actonparams

## END OF Main ##
